package com.sherler.crawler.eastmoney;

import com.sherler.crawler.eastmoney.jpa.GubaNews;
import com.sherler.crawler.eastmoney.jpa.GubaNewsRepository;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.context.annotation.ComponentScan;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.text.SimpleDateFormat;
import java.util.*;

/**
 * Created by GeorgeSherler on 2017/4/3.
 */
@Component
public class GubaPageProcessor implements PageProcessor{

    private Site site= Site.me().setRetryTimes(5).setSleepTime(1000).setCharset("utf-8").setTimeOut(20000).setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36");

    private ArrayList<String> urls=new ArrayList<>();

    private Integer currentPageNum=1;

    private String category;

    private int period=1;

    private boolean shouldGoOn=true;

    @Autowired
    private GubaNewsRepository gubaNewsRepository;

    private String primitiveUrl;

    public void start(int period,String url,String category){
        this.period=period;
        this.primitiveUrl=url;
        this.category=category;
        List<String[]> listProxyies=new ArrayList<>();
//        listProxyies.add(new String[]{"HP0B0J426G6PX5UP","348A76AA52FB9D0D","139.129.199.35","9010"});
//        listProxyies.add(new String[]{"HY56AQ0AZ77USTFP","93C6643AF55E4C8C","139.129.199.35","9010"});
//        listProxyies.add(new String[]{"H67FTUJJF5728S1P","31326E932EF1B748","139.129.199.35","9010"});
//        listProxyies.add(new String[]{"H4D39KK56GD5D16P","DE81E39203B2EC85","139.129.199.35","9010"});
//        listProxyies.add(new String[]{"H93XDR0U501E561P","8638650F275D5E21","139.129.199.35","9010"});
//        this.site.setHttpProxyPool(listProxyies,false);
        if(this.urls!=null && primitiveUrl!=null && category !=null)
        {
            this.category=category;
            synchronized (GubaPageProcessor.class){
                urls.add(primitiveUrl);
                while (urls.size()>0){
                    try {
                        Spider.create(this).addUrl().addUrl(urls.iterator().next()).thread(5).run();
                    }catch (Exception e){
                        System.out.println(e);
                    }
                }
            }
        }
    }

    @Override
    public void process(Page page) {
        if(this.urls!=null)
        {
            this.urls.remove(page.getRequest().getUrl());
        }

        if(isListPage(page))
        {
            String url=page.getRequest().getUrl();
            String[] pageUrlSegments=url.split(",|\\.");
            String pageInfo=pageUrlSegments[pageUrlSegments.length-2].replace("f_","");
            try{
                currentPageNum=Integer.parseInt(pageInfo);
            }catch (Exception e){
                System.out.println(e);
            }
            System.out.println("extractURLList:"+page.getUrl());
            extractURLList(page);
        }else{
            System.out.println("extractDocument:>>>>"+page.getUrl());
            extractContent(page);
        }
        if(this.urls!=null)
        {
            if(urls.size()>0){
                page.addTargetRequest(urls.iterator().next());
            }
        }
    }

    /**
     * 提取列表页的URL
     * @param page
     */
    private void extractURLList(Page page){
        List<String> paperUrls=page.getHtml().xpath("//ul[@class='newlist']/li/span[@class='sub']/a[2]").links().all();
        for(String url:paperUrls){
            if(urls.contains(url)){

            }else {
                GubaNews gubaNews=gubaNewsRepository.findOne(extractId(url));
                if(gubaNews==null||gubaNews.getId()==null){
                    urls.add(url);
                }
            }
        }
        if(shouldGoOn){
            int start =primitiveUrl.indexOf("f_");
            int end = primitiveUrl.indexOf(".html");
            String pageStr=primitiveUrl.substring(start,end);
            String listUrl = primitiveUrl.replace(pageStr,"f_"+(currentPageNum+1));
            System.out.println("new list url added >>>>>"+listUrl);
            urls.add(listUrl);
        }
    }

    private boolean crawledBefore(String id){
        GubaNews gubaNews=gubaNewsRepository.findOne(id);
        if(gubaNews!=null && gubaNews.getId()!=null)
        {
            return true;
        }
        return false;
    }

    private String extractId(String url){
        String[] urlSegments=url.split("/");
        return urlSegments[urlSegments.length-1].replace(".html","");
    }

    /**
     * 提取正文内容
     * @param page
     */
    private void extractContent(Page page){
        GubaNews gubaNews=new GubaNews();
        gubaNews.setBarName(page.getHtml().xpath("//div[@id='stockheader']/span/span/allText()").toString());

        int visitsStart=page.getRawText().indexOf("num=")+4;
        String visits=page.getRawText().substring(visitsStart,page.getRawText().indexOf(";",visitsStart));
        gubaNews.setVisitCount(visits);

        int commentStart=page.getRawText().indexOf("pinglun_num=")+12;
        String comments=page.getRawText().substring(commentStart,page.getRawText().indexOf(";",commentStart));
        gubaNews.setCommentCount(comments);

        String content=page.getHtml().xpath("//div[@id='zw_body']/allText()").toString();
        if(category.equals("问董秘"))
        {
            content = page.getHtml().xpath("//div[@class='zwcontentmain']//div[@class='answer_wrap']//div[@class='content_wrap']/allText()").toString();
        }
        if(content!=null)
        {
            gubaNews.setContent(content.trim());
        }

        String title=page.getHtml().xpath("//div[@id='zwconttbt']/allText()").toString();
        if(category.equals("问董秘"))
        {
            title = page.getHtml().xpath("//div[@class='zwcontentmain']//div[@class='question']/allText()").toString();
        }

        if(title!=null)
        {
            gubaNews.setTitle(title.trim());
        }

        List<String> keyStocks=page.getHtml().xpath("//div[@id='zw_body']//a/allText()").all();
        if(keyStocks!=null)
        {
            String keyStockString= StringUtils.join(keyStocks,";");
            gubaNews.setKeywords(keyStockString);
        }
        gubaNews.setUrl(page.getRequest().getUrl());
        gubaNews.setId(extractId(page.getRequest().getUrl()));

        String dateInfStr=page.getHtml().xpath("//div[@id='zwconttb']/div[2]/allText()").toString();
        if(dateInfStr==null)
        {
            dateInfStr= page.getHtml().xpath("//div[@id='zwconttb']/div[1]/allText()").toString();
        }
        if(dateInfStr!=null)
        {
            String[] dateInfo = dateInfStr.split("\\s+");
            String dataStr=dateInfo[1]+" "+dateInfo[2];
            SimpleDateFormat sdf =   new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss" );
            try{
                Date date =sdf.parse(dataStr);
                if(datesBetween(date,new Date())>period){
                    shouldGoOn=false;
                }
            }catch (Exception e){
                System.out.println(e);
            }
            gubaNews.setNewsDate(dataStr);
        }

        gubaNews.setCategory(this.category);
        gubaNewsRepository.save(gubaNews);
    }

    /**
     * 计算两个时间之间的日期距离
     * @param date2
     * @param date1
     * @return
     */
    private int datesBetween(Date date1,Date date2){
        Calendar calendar = Calendar.getInstance();
        long time1=0;
        long time2=0;
        try{
           calendar.setTime(date1);
           time1=calendar.getTimeInMillis();
           calendar.setTime(date2);
           time2=calendar.getTimeInMillis();
        }catch (Exception e){
            System.out.println(e);
        }
        long between_days=(time2-time1)/(1000*3600*24);

        return Integer.parseInt(String.valueOf(between_days));
    }

    /**
     * 是否为列表页
     * @param page
     * @return
     */
    private boolean isListPage(Page page){
        if(page==null)
        {
            return false;
        }
        String url = page.getRequest().getUrl();
        if(url!=null&&url.contains(",f_")){
            return true;
        }else{
            return false;
        }
    }



    @Override
    public Site getSite() {
        return site;
    }
}
